/*
 *  Copyright 2010 INFN - APE group
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

#pragma once

#include <list>

#include <boost/shared_ptr.hpp>
#include <boost/noncopyable.hpp>
#include <boost/thread.hpp>
#include <boost/thread/condition.hpp>
#include <boost/thread/mutex.hpp>

#include <cuqu/cuqu.h>
#include <cuos/types.h>
#include <cuos/detail/syscall.h>
#include <cuos/detail/pinned_mem_pool.h>

namespace cuos
{
    namespace host {
        class runtime;
        typedef boost::shared_ptr<runtime> runtime_ref_t;

        /*! The \p cuda_init function selects a CUDA device,
         * compatible with the cuOS requirements, which is either the
         * \p device_id or the first available one, if \p device_id is
         * equal to -1.
         *
         *  \param device_id Either the device index of the requested
         *  device or -1.
         *  \return SUCCESS or INVALID if no valid device is found.
         */
        int cuda_init(int device_id);

        runtime_ref_t make_runtime(int device_id, unsigned int block_size);

        // fwd decl
        class handler;

        class runtime: boost::noncopyable {
        public:
            typedef std::list<handler*> handler_queue_t;

        private:
            sys_req_queue_t m_queue_d2h;
            sys_rep_queue_t m_queue_h2d;
            int             m_device_id;
            cudaStream_t    m_io_stream;
            cudaStream_t    m_exec_stream;
            bool            m_prepared;
            sys_queue_t    *m_d_sys_q;
            handler_queue_t m_handler_queue;
            pinned_mem_pool m_pool;

            boost::condition m_cond;
            boost::thread *m_host_thread;
            int m_host_thread_started;
            volatile int m_host_thread_stop;
            boost::mutex m_mutex;


        protected:
            runtime(int device_id, unsigned int block_size);
            int handle_req(sys_req_t *req, sys_rep_t *rep);
            int handle_mpi_send(sys_req_t *req, sys_rep_t *rep);
            int handle_mpi_recv(sys_req_t *req, sys_rep_t *rep);
            void allocate_sys_q();

            //handler *make_handler(sys_req_t *req);
            void queue_handler(handler *h) { assert(h); m_handler_queue.push_back(h); }
            int progress_handlers();

            static void sys_server(runtime &);

        public:
            ~runtime();
            enum {
                timeout_infinite = -1,
                reply_timeout = 10000
            };
            // handle a single system service
            // always blocks for at least timeout_ms, unless timeout_ms == 0
            // to be called by CUDA device thread
            int handle_system_service(int timeout_ms = timeout_infinite);
            int handle_system_service_old(int timeout_ms = timeout_infinite);
            int prepare_device();
            cudaStream_t exec_stream();
            sys_queue_t *sys_q();
            int push_reply(sys_rep_t *rep);
            void *get_tmp_buf(size_t size);
            void rel_tmp_buf(void *buf);
            cudaStream_t io_stream();
            void start_sys_server();
            void stop_sys_server();

            friend runtime_ref_t make_runtime(int device_id, unsigned int block_size);
        };

    } // end namespace host
    
} // end namespace cuos

/*
 * Local variables:
 *  mode: c++
 *  c-indent-level: 4
 *  c-basic-offset: 4
 *  tab-width: 4
 *  indent-tabs-mode: nil
 * End:
 */
